import requests
from bs4 import BeautifulSoup
import csv

def getHTMLText(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return "获取文本异常"

def fillUnivList(ulist,soup):
    # data = soup.find('div', attrs='listleft').find_all('div')
    data = soup.find_all('div', attrs='leftcontent')

    for divs in data:
        contents = divs.find('div',attrs='bcontent')
        # month = divs.find('em').text.strip()
        day = divs.find('span').text.strip()
        title = divs.find('h3').text.strip()
        url= divs.find('a')['href'].replace('../../..','')
        urls='http://www.ccit.js.cn'+url
        # context = divs.find('p').text.strip().replace('\n','').replace('\r','')
        context = contents.text.strip().replace('\n','').replace('\r','')
        # date = month + day + "日"
        ulist.append([day,urls,title,context])



def writedatatocsv(ulist):
    with open('data.csv', 'a', newline='', encoding='utf-8') as file:
        writer = csv.writer(file)
        num = len(ulist)
        for i in range(num):
            u = ulist[i]
            writer.writerow([u[0],u[1],u[2],u[3]])


# def main():
#     ulist=[]
#     url='http://www.ccit.js.cn/zjcx/xyzx1/cxyw.htm'
#     html = getHTMLText(url)
#     soup = BeautifulSoup(html, "html.parser")
#     fillUnivList(ulist, soup)
#     writedatatocsv(ulist)

def main():
    for i in range(661):
        if(i == 0):
            url = 'http://www.ccit.js.cn/zjcx/xyzx1/cxyw.htm'
            with open('data.csv', 'a', encoding='utf-8') as file:
                writer = csv.writer(file)
                writer.writerow(["日期","网址", "标题", "内容"])
        else:
             url = 'http://www.ccit.js.cn/zjcx/xyzx1/cxyw/' + str(661-i) +'.htm'
        ulist = []
        html = getHTMLText(url)
        soup = BeautifulSoup(html, "html.parser")
        fillUnivList(ulist, soup)
        writedatatocsv(ulist)



main()